import re
import os
import time
import threading
from multiprocessing import Pool, cpu_count

import requests
from bs4 import BeautifulSoup

HEADERS = {
    'X-Requested-With': 'XMLHttpRequest',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
                  '(KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
    'Referer': 'http://www.dianping.com'
}

def get_hot_regions():
	r = requests.get("http://www.dianping.com/huhehaote/food", headers=HEADERS, timeout=10).text
	hot_regions = BeautifulSoup(r, 'lxml').find('ul', class_="nc_list").find_all('li')
	# print(len(hot_regions))
	with open("hot_regions.txt", 'w') as f:
		f.write("[")
		for hot_region in hot_regions:
			# print(hot_region.find('a').get_text())
			f.write("'" + hot_region.find('a').get_text() + "',")
		f.write("]")
		
def get_regions():
	req = requests.get("http://www.dianping.com/huhehaote/food", headers=HEADERS, timeout=10).text
	regions_text = BeautifulSoup(req, 'lxml').find("script", class_="J_auto-load").get_text()
	soup = BeautifulSoup(str(regions_text), "html.parser")
	regions = soup.find_all('dl', class_="list")
	with open("regions.txt", 'w') as f:
		f.write("[\n")
		for region in regions: 
			# print(region.find('dt').find('a').get_text())
			f.write("'" + region.find('dt').find('a').get_text() + "':[")
			items = region.find("dd").find('ul').find_all('li')
			for item in items:
				f.write("'" + item.find('a').get_text() + "',")
				# print(item.find('a').get_text())
			f.write("],\n")
		f.write("]")
		
		
def get_cooking():
	req = requests.get("http://www.dianping.com/huhehaote/food", headers=HEADERS, timeout=10).text
	soup = BeautifulSoup(req, 'lxml').find('textarea', class_='J_auto-load').get_text()
	print(soup)
	
	
if __name__=="__main__":
	# get_hot_regions()
	# get_regions()
	get_cooking()